{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2022-01-23 08:19:30--  https://mirror.coggle.club/dataset/afqmc_public.zip\n",
      "正在解析主机 mirror.coggle.club (mirror.coggle.club)... 124.95.157.146, 2408:8722:1010:1:3::3fb\n",
      "正在连接 mirror.coggle.club (mirror.coggle.club)|124.95.157.146|:443... 已连接。\n",
      "已发出 HTTP 请求，正在等待回应... 200 OK\n",
      "长度： 1195044 (1.1M) [application/zip]\n",
      "正在保存至: “afqmc_public.zip”\n",
      "\n",
      "afqmc_public.zip    100%[===================>]   1.14M   323KB/s    用时 3.6s    \n",
      "\n",
      "2022-01-23 08:19:35 (323 KB/s) - 已保存 “afqmc_public.zip” [1195044/1195044])\n",
      "\n",
      "Archive:  afqmc_public.zip\n",
      "  inflating: afqmc_public/train.json  \n",
      "  inflating: afqmc_public/test.json  \n",
      "  inflating: afqmc_public/dev.json   \n"
     ]
    }
   ],
   "source": [
    "# !wget https://mirror.coggle.club/dataset/afqmc_public.zip\n",
    "# !unzip afqmc_public.zip -d afqmc_public"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence1</th>\n",
       "      <th>sentence2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>蚂蚁借呗等额还款可以换成先息后本吗</td>\n",
       "      <td>借呗有先息到期还本吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>蚂蚁花呗说我违约一次</td>\n",
       "      <td>蚂蚁花呗违约行为是什么</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>帮我看一下本月花呗账单有没有结清</td>\n",
       "      <td>下月花呗账单</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>蚂蚁借呗多长时间综合评估一次</td>\n",
       "      <td>借呗得评估多久</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>我的花呗账单是***，还款怎么是***</td>\n",
       "      <td>我的花呗，月结出来说让我还***元，我自己算了一下详细名单我应该还***元</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38645</th>\n",
       "      <td>使用的借呗之后，可以出国吗，因为我的工作在国外，我需要在国外还款</td>\n",
       "      <td>借呗这个月分期还款了，还了的钱还可以再次取出用吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38646</th>\n",
       "      <td>还要多久能开通花呗</td>\n",
       "      <td>花呗要多久才会重新开通</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38647</th>\n",
       "      <td>怎样设置花呗访问手机通讯录</td>\n",
       "      <td>花呗读取不了手机通讯录权限</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38648</th>\n",
       "      <td>花呗的学历填错了，可不可以改</td>\n",
       "      <td>花呗学历填错了怎样修改</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38649</th>\n",
       "      <td>我的淘宝绑不了花呗</td>\n",
       "      <td>怎么花呗绑不了淘宝</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>38650 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              sentence1  \\\n",
       "0                     蚂蚁借呗等额还款可以换成先息后本吗   \n",
       "1                            蚂蚁花呗说我违约一次   \n",
       "2                      帮我看一下本月花呗账单有没有结清   \n",
       "3                        蚂蚁借呗多长时间综合评估一次   \n",
       "4                   我的花呗账单是***，还款怎么是***   \n",
       "...                                 ...   \n",
       "38645  使用的借呗之后，可以出国吗，因为我的工作在国外，我需要在国外还款   \n",
       "38646                         还要多久能开通花呗   \n",
       "38647                     怎样设置花呗访问手机通讯录   \n",
       "38648                    花呗的学历填错了，可不可以改   \n",
       "38649                         我的淘宝绑不了花呗   \n",
       "\n",
       "                                   sentence2 label  \n",
       "0                                 借呗有先息到期还本吗     0  \n",
       "1                                蚂蚁花呗违约行为是什么     0  \n",
       "2                                     下月花呗账单     0  \n",
       "3                                    借呗得评估多久     0  \n",
       "4      我的花呗，月结出来说让我还***元，我自己算了一下详细名单我应该还***元     1  \n",
       "...                                      ...   ...  \n",
       "38645               借呗这个月分期还款了，还了的钱还可以再次取出用吗     0  \n",
       "38646                            花呗要多久才会重新开通     0  \n",
       "38647                          花呗读取不了手机通讯录权限     0  \n",
       "38648                            花呗学历填错了怎样修改     0  \n",
       "38649                              怎么花呗绑不了淘宝     1  \n",
       "\n",
       "[38650 rows x 3 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os, json, codecs\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import jieba, gensim\n",
    "%pylab inline\n",
    "import seaborn as sbs\n",
    "\n",
    "train_json = codecs.open('afqmc_public/train.json').readlines()\n",
    "train_json = [json.loads(x) for x in train_json]\n",
    "\n",
    "dev_json = codecs.open('afqmc_public/dev.json').readlines()\n",
    "dev_json = [json.loads(x) for x in dev_json]\n",
    "\n",
    "train_data = pd.DataFrame(train_json + dev_json)\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache /tmp/jieba.cache\n",
      "Loading model cost 0.781 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    }
   ],
   "source": [
    "train_sentence1 = train_data['sentence1'].apply(lambda x: list(jieba.cut(x.replace('*', ''))))\n",
    "train_sentence2 = train_data['sentence2'].apply(lambda x: list(jieba.cut(x.replace('*', ''))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                  [蚂蚁, 借, 呗, 等额, 还款, 可以, 换成, 先息, 后, 本, 吗]\n",
       "1                                 [蚂蚁, 花, 呗, 说, 我, 违约, 一次]\n",
       "2                     [帮, 我, 看, 一下, 本月, 花, 呗, 账单, 有没有, 结清]\n",
       "3                             [蚂蚁, 借, 呗, 多长时间, 综合, 评估, 一次]\n",
       "4                        [我, 的, 花, 呗, 账单, 是, ，, 还款, 怎么, 是]\n",
       "                               ...                        \n",
       "38645    [使用, 的, 借, 呗, 之后, ，, 可以, 出国, 吗, ，, 因为, 我, 的, 工...\n",
       "38646                                [还要, 多久, 能, 开通, 花, 呗]\n",
       "38647                          [怎样, 设置, 花, 呗, 访问, 手机, 通讯录]\n",
       "38648                     [花, 呗, 的, 学历, 填错, 了, ，, 可不可以, 改]\n",
       "38649                              [我, 的, 淘宝, 绑, 不了, 花, 呗]\n",
       "Name: sentence1, Length: 38650, dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sentence1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = gensim.models.Word2Vec(sentences=train_sentence1 + train_sentence2, \n",
    "                                   vector_size=100, window=5, min_count=1, workers=4)\n",
    "model.save(\"word2vec.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('顾客', 0.9444351196289062),\n",
       " ('客人', 0.9340370893478394),\n",
       " ('买家', 0.9080636501312256),\n",
       " ('口碑', 0.9019301533699036),\n",
       " ('店铺', 0.8943435549736023),\n",
       " ('商户', 0.8719239830970764),\n",
       " ('扫', 0.8416370749473572),\n",
       " ('向', 0.8202187418937683),\n",
       " ('当面', 0.8193756341934204),\n",
       " ('人家', 0.8153225183486938)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.most_similar('客户')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.22285709, -0.15922348, -0.09186552,  0.80624914, -0.4646926 ,\n",
       "       -0.558344  ,  0.5610685 ,  0.42486352, -0.5246372 , -0.30065185,\n",
       "       -0.58015746, -0.9088494 ,  0.32266712,  0.680608  , -0.13058656,\n",
       "       -0.22997667,  0.1047236 , -0.38451314,  0.03001483, -0.7118729 ,\n",
       "        0.7983152 ,  0.39554375,  0.39508587,  0.81738466,  0.28954676,\n",
       "        0.55993956,  0.16604969, -0.06719798, -0.11952766,  0.07739814,\n",
       "        0.03282386,  0.05381026,  0.7517788 , -0.23061617,  0.41787574,\n",
       "        0.39359957,  0.29080775, -0.7472741 , -0.52234954, -0.4435867 ,\n",
       "        0.31004262, -0.75693905, -0.01570158, -0.21923868,  0.15152332,\n",
       "        0.04119374, -0.5509554 , -0.31096122,  0.11871855, -1.0280432 ,\n",
       "        0.29447117, -0.15362723, -0.16881391, -0.3225671 , -0.05096226,\n",
       "       -0.01893405,  0.16975991, -0.25489917, -0.25233012, -0.01053992,\n",
       "       -0.2450923 , -0.53756994, -0.27630597, -0.07334035, -0.19924028,\n",
       "        0.21671961,  0.2230083 ,  0.5888119 , -0.34663934,  0.2795648 ,\n",
       "       -0.73890644, -0.24925815, -0.3877863 , -0.2565886 ,  0.47968745,\n",
       "        0.26714066,  0.3222914 ,  0.38155112,  0.44006744, -0.21632375,\n",
       "       -0.24629763,  0.29850537,  0.01832781, -0.42271307, -0.05307665,\n",
       "       -0.32105985,  0.68007845, -0.28007576,  0.579378  , -0.22036637,\n",
       "        0.06423527,  0.46775752, -0.621488  , -0.4253591 , -0.2488479 ,\n",
       "        0.31009188, -0.48619446, -0.09184262,  0.08720241,  0.6204805 ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv['客户']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 序列编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第一个样本的第一个句子\n",
    "arr1 = np.vstack([model.wv[x] for x in train_sentence1[0]])\n",
    "\n",
    "# 第一个样本的第二个句子\n",
    "arr2 = np.vstack([model.wv[x] for x in train_sentence2[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((11, 100), (7, 100))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr1.shape, arr2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_sentence(sentence):\n",
    "    arr = []\n",
    "    for x in sentence:\n",
    "        arr.append(model.wv[x])\n",
    "        \n",
    "    if len(arr) == 0:\n",
    "        return np.zeros(shape=(1, 100))\n",
    "    else:\n",
    "        return np.vstack(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_feat = [ encode_sentence(x) for x in train_sentence1]\n",
    "train_sentence2_feat = [ encode_sentence(x) for x in train_sentence2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## mean-pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_meanfeat = np.vstack([x.mean(0) for x in train_sentence1_feat])\n",
    "train_sentence2_meanfeat = np.vstack([x.mean(0) for x in train_sentence2_feat])\n",
    "\n",
    "from sklearn.preprocessing import normalize\n",
    "train_sentence1_meanfeat = normalize(train_sentence1_meanfeat)\n",
    "train_sentence2_meanfeat = normalize(train_sentence2_meanfeat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.857872283257604"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sentence1_meanfeat[0].dot(train_sentence2_meanfeat[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_pooing_distance = [np.dot(x, y) for x, y in \n",
    "                        zip(train_sentence1_meanfeat, train_sentence2_meanfeat)]\n",
    "\n",
    "# 所有样本的 句子1 与 句子2 的mean-pooing的相似度\n",
    "# 句子1 -》 100\n",
    "# 句子2 -》 100\n",
    "train_data['mean_pooing'] = mean_pooing_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='label'>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPNElEQVR4nO3dfYxdeV3H8feHLpUISIi9EuzDtpEilIcIOxaVBEGW2IWk1bCaVo2sQRojBRRiKJE0pEYjGOEPUwyNbiAIlLp/mFFGKuFBAwLOLCyLbS1M6kJbTRiWBUIMdAtf/5i7cLmdmXumvTOz/c37lUxyzzm/vfebzeS9Z8+9Z26qCknSje8Raz2AJGk8DLokNcKgS1IjDLokNcKgS1IjDLokNeKmtXrhTZs21fbt29fq5SXphnT33Xd/tap6Cx1bs6Bv376dmZmZtXp5SbohJfnSYse85CJJjTDoktQIgy5JjTDoktQIgy5JjTDoktQIgy5JjTDoktSINbuxSNL12X74A2s9QlPu+/OXrPUI180zdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhJ9DH8HP+o5XC5/1lR6uOp2hJ9mT5FyS2SSHFzi+LclHk3w2yb1JXjz+USVJSxkZ9CQbgGPAbcAu4ECSXUPL3gicrKpnAfuBt497UEnS0rqcoe8GZqvqfFVdBk4A+4bWFPBj/cePA/5nfCNKkrrocg19M3BhYPsi8JyhNW8C/iXJq4BHA7eOZTpJUmfj+pTLAeCdVbUFeDHw7iRXPXeSg0lmkszMzc2N6aUlSdAt6JeArQPbW/r7Br0cOAlQVZ8EHgVsGn6iqjpeVRNVNdHr9a5tYknSgroEfRrYmWRHko3Mv+k5ObTmy8ALAZI8lfmgewouSatoZNCr6gpwCDgFnGX+0yynkxxNsre/7HXAK5J8DngfcEdV1UoNLUm6Wqcbi6pqCpga2ndk4PEZ4LnjHU2StBze+i9JjTDoktQIgy5JjTDoktQIgy5JjTDoktQIgy5JjTDoktQIgy5JjTDoktQIgy5JjTDoktQIgy5JjTDoktQIgy5JjegU9CR7kpxLMpvk8ALH35bknv7PF5J8feyTSpKWNPILLpJsAI4BLwIuAtNJJvtfagFAVf3hwPpXAc9agVklSUvocoa+G5itqvNVdRk4AexbYv0B5r+GTpK0iroEfTNwYWD7Yn/fVZLcDOwAPnL9o0mSlmPcb4ruB+6qqu8udDDJwSQzSWbm5ubG/NKStL51CfolYOvA9pb+voXsZ4nLLVV1vKomqmqi1+t1n1KSNFKXoE8DO5PsSLKR+WhPDi9K8hTg8cAnxzuiJKmLkUGvqivAIeAUcBY4WVWnkxxNsndg6X7gRFXVyowqSVrKyI8tAlTVFDA1tO/I0PabxjeWJGm5vFNUkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEZ2CnmRPknNJZpMcXmTNryc5k+R0kveOd0xJ0igjv7EoyQbgGPAi4CIwnWSyqs4MrNkJvAF4blU9kOQnVmpgSdLCupyh7wZmq+p8VV0GTgD7hta8AjhWVQ8AVNVXxjumJGmULkHfDFwY2L7Y3zfoycCTk3wiyaeS7BnXgJKkbjp9SXTH59kJPB/YAvxbkmdU1dcHFyU5CBwE2LZt25heWpIE3c7QLwFbB7a39PcNughMVtWDVfXfwBeYD/wPqarjVTVRVRO9Xu9aZ5YkLaBL0KeBnUl2JNkI7Acmh9b8A/Nn5yTZxPwlmPPjG1OSNMrIoFfVFeAQcAo4C5ysqtNJjibZ2192Crg/yRngo8AfVdX9KzW0JOlqna6hV9UUMDW078jA4wJe2/+RJK0B7xSVpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEZ0CnqSPUnOJZlNcniB43ckmUtyT//nd8c/qiRpKSO/4CLJBuAY8CLmvzt0OslkVZ0ZWvr+qjq0AjNKkjrocoa+G5itqvNVdRk4Aexb2bEkScvVJeibgQsD2xf7+4a9NMm9Se5KsnUs00mSOhvXm6L/CGyvqmcCHwLetdCiJAeTzCSZmZubG9NLS5KgW9AvAYNn3Fv6+76vqu6vqu/0N/8GuGWhJ6qq41U1UVUTvV7vWuaVJC2iS9CngZ1JdiTZCOwHJgcXJHniwOZe4Oz4RpQkdTHyUy5VdSXJIeAUsAG4s6pOJzkKzFTVJPDqJHuBK8DXgDtWcGZJ0gJGBh2gqqaAqaF9RwYevwF4w3hHkyQth3eKSlIjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjOgU9yZ4k55LMJjm8xLqXJqkkE+MbUZLUxcigJ9kAHANuA3YBB5LsWmDdY4HXAJ8e95CSpNG6nKHvBmar6nxVXQZOAPsWWPcnwJuBb49xPklSR12Cvhm4MLB9sb/v+5I8G9haVR8Y42ySpGW47jdFkzwCeCvwug5rDyaZSTIzNzd3vS8tSRrQJeiXgK0D21v6+x7yWODpwMeS3Af8HDC50BujVXW8qiaqaqLX61371JKkq3QJ+jSwM8mOJBuB/cDkQwer6htVtamqtlfVduBTwN6qmlmRiSVJCxoZ9Kq6AhwCTgFngZNVdTrJ0SR7V3pASVI3N3VZVFVTwNTQviOLrH3+9Y8lSVou7xSVpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqRKegJ9mT5FyS2SSHFzj+e0k+n+SeJB9Psmv8o0qSljIy6Ek2AMeA24BdwIEFgv3eqnpGVf0M8BbgreMeVJK0tC5n6LuB2ao6X1WXgRPAvsEFVfXNgc1HAzW+ESVJXXT5TtHNwIWB7YvAc4YXJXkl8FpgI/BLCz1RkoPAQYBt27Ytd1ZJ0hLG9qZoVR2rqp8CXg+8cZE1x6tqoqomer3euF5akkS3oF8Ctg5sb+nvW8wJ4FeuYyZJ0jXoEvRpYGeSHUk2AvuBycEFSXYObL4E+OL4RpQkdTHyGnpVXUlyCDgFbADurKrTSY4CM1U1CRxKcivwIPAA8LKVHFqSdLUub4pSVVPA1NC+IwOPXzPmuSRJy+SdopLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUiE5BT7Inybkks0kOL3D8tUnOJLk3yYeT3Dz+USVJSxkZ9CQbgGPAbcAu4ECSXUPLPgtMVNUzgbuAt4x7UEnS0rqcoe8GZqvqfFVdZv5LoPcNLqiqj1bV//U3P8X8F0lLklZRl6BvBi4MbF/s71vMy4F/vp6hJEnL1+k7RbtK8lvABPCLixw/CBwE2LZt2zhfWpLWvS5n6JeArQPbW/r7fkiSW4E/BvZW1XcWeqKqOl5VE1U10ev1rmVeSdIiugR9GtiZZEeSjcB+YHJwQZJnAe9gPuZfGf+YkqRRRga9qq4Ah4BTwFngZFWdTnI0yd7+sr8AHgP8fZJ7kkwu8nSSpBXS6Rp6VU0BU0P7jgw8vnXMc0mSlsk7RSWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEZ2CnmRPknNJZpMcXuD485J8JsmVJLePf0xJ0igjg55kA3AMuA3YBRxIsmto2ZeBO4D3jntASVI3Xb6xaDcwW1XnAZKcAPYBZx5aUFX39Y99bwVmlCR10OWSy2bgwsD2xf4+SdLDyKq+KZrkYJKZJDNzc3Or+dKS1LwuQb8EbB3Y3tLft2xVdbyqJqpqotfrXctTSJIW0SXo08DOJDuSbAT2A5MrO5YkablGBr2qrgCHgFPAWeBkVZ1OcjTJXoAkP5vkIvBrwDuSnF7JoSVJV+vyKReqagqYGtp3ZODxNPOXYiRJa8Q7RSWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhrRKehJ9iQ5l2Q2yeEFjv9Ikvf3j386yfaxTypJWtLIoCfZABwDbgN2AQeS7Bpa9nLggap6EvA24M3jHlSStLQuZ+i7gdmqOl9Vl4ETwL6hNfuAd/Uf3wW8MEnGN6YkaZQu3ym6GbgwsH0ReM5ia6rqSpJvAD8OfHVwUZKDwMH+5reSnLuWobWgTQz9+344iv/vth75uzleNy92oNOXRI9LVR0Hjq/ma64XSWaqamKt55CG+bu5erpccrkEbB3Y3tLft+CaJDcBjwPuH8eAkqRuugR9GtiZZEeSjcB+YHJozSTwsv7j24GPVFWNb0xJ0igjL7n0r4kfAk4BG4A7q+p0kqPATFVNAn8LvDvJLPA15qOv1eWlLD1c+bu5SuKJtCS1wTtFJakRBl2SGmHQJakRq/o5dI1Hkqcwf3fu5v6uS8BkVZ1du6kkrTXP0G8wSV7P/J9fCPAf/Z8A71voD6dJDxdJfmetZ2idn3K5wST5AvC0qnpwaP9G4HRV7VybyaSlJflyVW1b6zla5iWXG8/3gJ8EvjS0/4n9Y9KaSXLvYoeAJ6zmLOuRQb/x/AHw4SRf5Ad/NG0b8CTg0FoNJfU9Afhl4IGh/QH+ffXHWV8M+g2mqj6Y5MnM/1njwTdFp6vqu2s3mQTAPwGPqap7hg8k+diqT7POeA1dkhrhp1wkqREGXZIaYdC1LiT51ojj25P85zKf851Jbr++yaTxMeiS1AiDrnUlyWOSfDjJZ5J8PsngF57flOQ9Sc4muSvJj/b/mVuS/GuSu5OcSvLENRpfWpJB13rzbeBXq+rZwAuAv0yS/rGfBt5eVU8Fvgn8fpJHAn8F3F5VtwB3An+6BnNLI/k5dK03Af4syfOYv7N2Mz+4g/FCVX2i//jvgFcDHwSeDnyo3/0NwP+u6sRSRwZd681vAj3glqp6MMl9wKP6x4Zvyijm/wNwuqp+fvVGlK6Nl1y03jwO+Eo/5i8Abh44ti3JQ+H+DeDjwDmg99D+JI9M8rRVnVjqyKBrvXkPMJHk88BvA/81cOwc8MokZ4HHA39dVZeB24E3J/kccA/wC6s7stSNt/5LUiM8Q5ekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWrE/wM6Oql9WR3A1wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_data.groupby('label')['mean_pooing'].mean().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## max-pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_meanfeat = np.vstack([x.max(0) for x in train_sentence1_feat])\n",
    "train_sentence2_meanfeat = np.vstack([x.max(0) for x in train_sentence2_feat])\n",
    "\n",
    "from sklearn.preprocessing import normalize\n",
    "train_sentence1_meanfeat = normalize(train_sentence1_meanfeat)\n",
    "train_sentence2_meanfeat = normalize(train_sentence2_meanfeat)\n",
    "\n",
    "max_pooing_distance = [np.dot(x, y) for x, y in \n",
    "                        zip(train_sentence1_meanfeat, train_sentence2_meanfeat)]\n",
    "train_data['max_pooing'] = max_pooing_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='label'>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAALu0lEQVR4nO3db4id+VmH8evbpLHI1lXMWGqS3Qk0q02L0O4Q/xS0pRWzFRLERRIt/mFpXmj8Q4sYURaJKK5FfSGpGLFUam3c7gsZbGyQdStY3ZpZu92axNQhbptEYafrUilSs9HbF3PWHs/OzDnZnMxs7rk+MHCe5/nlnJswXHnmOeeZpKqQJN3+XrHRA0iSpsOgS1ITBl2SmjDoktSEQZekJgy6JDWxdaNeePv27TU7O7tRLy9Jt6Unnnjii1U1s9KxDQv67OwsCwsLG/XyknRbSvL51Y55yUWSmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMbdmORpJsze+xjGz1CK0//xvdv9Ag3zTN0SWrCoEtSE15yGcMfa6erw4+10suVZ+iS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpiYmCnmR/kotJFpMcW+H4XUkeS/LpJE8leef0R5UkrWVs0JNsAU4A9wF7gcNJ9o4s+2Xg4ap6E3AIeP+0B5UkrW2SM/R9wGJVXaqqa8Ap4ODImgK+bvD4TuBfpzeiJGkSkwR9B3B5aPvKYN+wXwHeleQKcBr46ZWeKMmRJAtJFpaWll7CuJKk1UzrTdHDwAeraifwTuBDSV703FV1sqrmqmpuZmZmSi8tSYLJgn4V2DW0vXOwb9gDwMMAVfV3wKuA7dMYUJI0mUmCfhbYk2R3km0sv+k5P7LmC8DbAZK8nuWge01FktbR2KBX1XXgKHAGuMDyp1nOJTme5MBg2XuBdyf5DPAR4Merqm7V0JKkF9s6yaKqOs3ym53D+x4cenweeMt0R5Mk3QjvFJWkJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTUwU9CT7k1xMspjk2CprfijJ+STnkvzJdMeUJI2zddyCJFuAE8D3AleAs0nmq+r80Jo9wC8Cb6mq55J8060aWJK0sknO0PcBi1V1qaquAaeAgyNr3g2cqKrnAKrqmemOKUkaZ5Kg7wAuD21fGewbdg9wT5JPJnk8yf6VnijJkSQLSRaWlpZe2sSSpBVN603RrcAe4K3AYeAPknz96KKqOllVc1U1NzMzM6WXliTBZEG/Cuwa2t452DfsCjBfVc9X1b8An2M58JKkdTJJ0M8Ce5LsTrINOATMj6z5M5bPzkmyneVLMJemN6YkaZyxQa+q68BR4AxwAXi4qs4lOZ7kwGDZGeDZJOeBx4Cfr6pnb9XQkqQXG/uxRYCqOg2cHtn34NDjAt4z+JIkbQDvFJWkJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTUwU9CT7k1xMspjk2BrrfjBJJZmb3oiSpEmMDXqSLcAJ4D5gL3A4yd4V1r0a+FngU9MeUpI03iRn6PuAxaq6VFXXgFPAwRXW/SrwEPCVKc4nSZrQJEHfAVwe2r4y2Pd/krwZ2FVVH1vriZIcSbKQZGFpaemGh5Ukre6m3xRN8grgt4H3jltbVSeraq6q5mZmZm72pSVJQyYJ+lVg19D2zsG+F7waeCPwiSRPA98BzPvGqCStr0mCfhbYk2R3km3AIWD+hYNV9aWq2l5Vs1U1CzwOHKiqhVsysSRpRWODXlXXgaPAGeAC8HBVnUtyPMmBWz2gJGkyWydZVFWngdMj+x5cZe1bb34sSdKN8k5RSWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktTEREFPsj/JxSSLSY6tcPw9Sc4neSrJo0nunv6okqS1jA16ki3ACeA+YC9wOMnekWWfBuaq6tuAR4DfnPagkqS1TXKGvg9YrKpLVXUNOAUcHF5QVY9V1X8ONh8Hdk53TEnSOJMEfQdweWj7ymDfah4A/mKlA0mOJFlIsrC0tDT5lJKksab6pmiSdwFzwPtWOl5VJ6tqrqrmZmZmpvnSkrTpbZ1gzVVg19D2zsG+/yfJO4BfAr6nqv5rOuNJkiY1yRn6WWBPkt1JtgGHgPnhBUneBPw+cKCqnpn+mJKkccYGvaquA0eBM8AF4OGqOpfkeJIDg2XvA+4APprkySTzqzydJOkWmeSSC1V1Gjg9su/BocfvmPJckqQb5J2iktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJamJiYKeZH+Si0kWkxxb4fjXJPnTwfFPJZmd+qSSpDWNDXqSLcAJ4D5gL3A4yd6RZQ8Az1XV64DfAR6a9qCSpLVNcoa+D1isqktVdQ04BRwcWXMQ+KPB40eAtyfJ9MaUJI2zdYI1O4DLQ9tXgG9fbU1VXU/yJeAbgS8OL0pyBDgy2PxykosvZWitaDsjf98vR/Fnt83I783punu1A5MEfWqq6iRwcj1fc7NIslBVcxs9hzTK7831M8kll6vArqHtnYN9K65JshW4E3h2GgNKkiYzSdDPAnuS7E6yDTgEzI+smQd+bPD4fuCvqqqmN6YkaZyxl1wG18SPAmeALcAHqupckuPAQlXNA38IfCjJIvDvLEdf68tLWXq58ntzncQTaUnqwTtFJakJgy5JTRh0SWpiXT+HrulI8q0s3527Y7DrKjBfVRc2bipJG80z9NtMkl9g+dcvBPj7wVeAj6z0i9Okl4skP7HRM3Tnp1xuM0k+B7yhqp4f2b8NOFdVezZmMmltSb5QVXdt9Bydecnl9vM/wDcDnx/Z/9rBMWnDJHlqtUPAa9Zzls3IoN9+fg54NMk/89VfmnYX8Drg6EYNJQ28Bvg+4LmR/QH+dv3H2VwM+m2mqj6e5B6Wf63x8JuiZ6vqvzduMgmAPwfuqKonRw8k+cS6T7PJeA1dkprwUy6S1IRBl6QmDLo2hSRfHnN8Nsk/3uBzfjDJ/Tc3mTQ9Bl2SmjDo2lSS3JHk0ST/kOSzSYb/w/OtST6c5EKSR5J87eDP3Jvkr5M8keRMktdu0PjSmgy6NpuvAD9QVW8G3gb8VpIMjn0L8P6qej3wH8BPJnkl8LvA/VV1L/AB4Nc2YG5pLD+Hrs0mwK8n+W6W76zdwVfvYLxcVZ8cPP5j4GeAjwNvBP5y0P0twL+t68TShAy6NpsfAWaAe6vq+SRPA68aHBu9KaNY/gfgXFV95/qNKL00XnLRZnMn8Mwg5m8D7h46dleSF8L9w8DfABeBmRf2J3llkjes68TShAy6NpsPA3NJPgv8KPBPQ8cuAj+V5ALwDcDvVdU14H7goSSfAZ4Evmt9R5Ym463/ktSEZ+iS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpr4X7ZNXuVcH5TvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_data.groupby('label')['max_pooing'].mean().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TFIDF Pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(max_features=100000)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tfidf = TfidfVectorizer(max_features=100000)\n",
    "tfidf.fit([' '.join(x) for x in train_sentence1 + train_sentence2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.521590339312075"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf.idf_[tfidf.vocabulary_['蚂蚁']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tfidf_pooing(sentence, sentence_word2vec):\n",
    "    weights = []\n",
    "    feats = []\n",
    "    for word,feat in zip(sentence, sentence_word2vec):\n",
    "        if word in tfidf.vocabulary_:\n",
    "            weights.append(tfidf.idf_[tfidf.vocabulary_[word]])\n",
    "            feats.append(feat)\n",
    "    weights = np.array(weights)\n",
    "    feats = np.array(feats)\n",
    "    \n",
    "    if len(feats) == 0:\n",
    "        return np.zeros(100)\n",
    "    else:\n",
    "        return np.matmul(weights, feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_tfidffeat = [tfidf_pooing(sentence, sentence_feat) \n",
    "    for sentence, sentence_feat in zip(train_sentence1[:], train_sentence1_feat[:])]\n",
    "\n",
    "train_sentence2_tfidffeat = [tfidf_pooing(sentence, sentence_feat) \n",
    "    for sentence, sentence_feat in zip(train_sentence2[:], train_sentence2_feat[:])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "train_sentence1_tfidffeat = np.vstack(train_sentence1_tfidffeat)\n",
    "train_sentence2_tfidffeat = np.vstack(train_sentence2_tfidffeat)\n",
    "\n",
    "train_sentence1_tfidffeat = normalize(train_sentence1_tfidffeat)\n",
    "train_sentence2_tfidffeat = normalize(train_sentence2_tfidffeat)\n",
    "\n",
    "tfidf_pooing_distance = [np.dot(x, y) for x, y in \n",
    "                        zip(train_sentence1_tfidffeat, train_sentence2_tfidffeat)]\n",
    "train_data['tfidf_pooing'] = tfidf_pooing_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='label'>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPMUlEQVR4nO3dfYxdeV3H8feHKZUIiMSOBPuwbaQI5SHCjkUlQZAldiVpNaymVSNrkMZIAYUYSiQNqdHwECHGFEOjGwgCpe4fZpSRSnjQgIAziwvY1sKkLnSqCcOyQIiBbuHrH3MXLndv5562d2a2v3m/kknuOee3936zmbz37Ln3zE1VIUm68T1srQeQJI2HQZekRhh0SWqEQZekRhh0SWqEQZekRmxYqxfetGlTbd++fa1eXpJuSHfddddXqmpy2LFOQU+yB/gLYAL466p6w8DxbcA7gR/trTlcVTPLPef27duZm5vr8vKSpJ4kX7zSsZGXXJJMAMeAW4FdwIEkuwaWvQ44WVXPAPYDb7v2cSVJ16LLNfTdwHxVna+qS8AJYN/AmgJ+pPf4McD/jG9ESVIXXS65bAYu9G0vAM8aWPN64J+TvBx4JHDLWKaTJHU2rk+5HADeUVVbgF8G3pXkQc+d5GCSuSRzi4uLY3ppSRJ0C/pFYGvf9pbevn4vAU4CVNUngEcAmwafqKqOV9VUVU1NTg59k1aSdI26BH0W2JlkR5KNLL3pOT2w5kvA8wGSPJmloHsKLkmraGTQq+oycAg4BZxl6dMsp5McTbK3t+zVwEuTfAZ4L3B7+Xd5JWlVdfoceu8z5TMD+470PT4DPHu8o0mSrsaa3Skq6fpsP/z+tR6hKfe84YVrPcJ182+5SFIjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcIvuBjBLxEYrxa+REB6qOp0hp5kT5JzSeaTHB5y/K1J7u79fD7J18Y+qSRpWSPP0JNMAMeAFwALwGyS6d73iAJQVX/Yt/7lwDNWYFZJ0jK6nKHvBuar6nxVXQJOAPuWWX8AeO84hpMkddcl6JuBC33bC719D5LkJmAH8OHrH02SdDXG/SmX/cCdVfWdYQeTHEwyl2RucXFxzC8tSetbl6BfBLb2bW/p7RtmP8tcbqmq41U1VVVTk5OT3aeUJI3UJeizwM4kO5JsZCna04OLkjwJeCzwifGOKEnqYmTQq+oycAg4BZwFTlbV6SRHk+ztW7ofOFFVtTKjSpKW0+nGoqqaAWYG9h0Z2H79+MaSJF0tb/2XpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEZ0CnqSPUnOJZlPcvgKa349yZkkp5O8Z7xjSpJGGfkVdEkmgGPAC4AFYDbJdFWd6VuzE3gt8Oyqui/Jj6/UwJKk4bqcoe8G5qvqfFVdAk4A+wbWvBQ4VlX3AVTVl8c7piRplC5B3wxc6Nte6O3r90TgiUk+nuSTSfaMa0BJUjcjL7lcxfPsBJ4LbAH+NcnTqupr/YuSHAQOAmzbtm1MLy1Jgm5n6BeBrX3bW3r7+i0A01V1f1X9N/B5lgL/A6rqeFVNVdXU5OTktc4sSRqiS9BngZ1JdiTZCOwHpgfW/D1LZ+ck2cTSJZjz4xtTkjTKyKBX1WXgEHAKOAucrKrTSY4m2dtbdgq4N8kZ4CPAH1XVvSs1tCTpwTpdQ6+qGWBmYN+RvscFvKr3I0laA94pKkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1IhOQU+yJ8m5JPNJDg85fnuSxSR3935+d/yjSpKWM/I7RZNMAMeAFwALwGyS6ao6M7D0fVV1aAVmlCR10OUMfTcwX1Xnq+oScALYt7JjSZKuVpegbwYu9G0v9PYNelGSzya5M8nWsUwnSepsXG+K/gOwvaqeDnwQeOewRUkOJplLMre4uDiml5YkQbegXwT6z7i39PZ9T1XdW1Xf7m3+NXDzsCeqquNVNVVVU5OTk9cyryTpCroEfRbYmWRHko3AfmC6f0GSx/dt7gXOjm9ESVIXIz/lUlWXkxwCTgETwB1VdTrJUWCuqqaBVyTZC1wGvgrcvoIzS5KGGBl0gKqaAWYG9h3pe/xa4LXjHU2SdDW8U1SSGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGtEp6En2JDmXZD7J4WXWvShJJZka34iSpC5GBj3JBHAMuBXYBRxIsmvIukcDrwQ+Ne4hJUmjdTlD3w3MV9X5qroEnAD2DVn3J8AbgW+NcT5JUkddgr4ZuNC3vdDb9z1Jnglsrar3j3E2SdJVuO43RZM8DHgL8OoOaw8mmUsyt7i4eL0vLUnq0yXoF4Gtfdtbevse8GjgqcBHk9wD/CwwPeyN0ao6XlVTVTU1OTl57VNLkh6kS9BngZ1JdiTZCOwHph84WFVfr6pNVbW9qrYDnwT2VtXcikwsSRpqZNCr6jJwCDgFnAVOVtXpJEeT7F3pASVJ3WzosqiqZoCZgX1HrrD2udc/liTpanmnqCQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiM6BT3JniTnkswnOTzk+O8l+VySu5N8LMmu8Y8qSVrOyKAnmQCOAbcCu4ADQ4L9nqp6WlX9NPAm4C3jHlSStLwuZ+i7gfmqOl9Vl4ATwL7+BVX1jb7NRwI1vhElSV1s6LBmM3Chb3sBeNbgoiQvA14FbAR+cSzTSZI6G9ubolV1rKp+EngN8Lpha5IcTDKXZG5xcXFcLy1JolvQLwJb+7a39PZdyQngV4YdqKrjVTVVVVOTk5Odh5QkjdYl6LPAziQ7kmwE9gPT/QuS7OzbfCHwhfGNKEnqYuQ19Kq6nOQQcAqYAO6oqtNJjgJzVTUNHEpyC3A/cB/w4pUcWpL0YF3eFKWqZoCZgX1H+h6/csxzSZKukneKSlIjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNaJT0JPsSXIuyXySw0OOvyrJmSSfTfKhJDeNf1RJ0nJGBj3JBHAMuBXYBRxIsmtg2X8AU1X1dOBO4E3jHlSStLwuZ+i7gfmqOl9Vl4ATwL7+BVX1kar6v97mJ4Et4x1TkjRKl6BvBi70bS/09l3JS4B/GnYgycEkc0nmFhcXu08pSRpprG+KJvktYAp487DjVXW8qqaqampycnKcLy1J696GDmsuAlv7trf09v2AJLcAfwz8QlV9ezzjSZK66nKGPgvsTLIjyUZgPzDdvyDJM4C3A3ur6svjH1OSNMrIoFfVZeAQcAo4C5ysqtNJjibZ21v2ZuBRwN8luTvJ9BWeTpK0QrpccqGqZoCZgX1H+h7fMua5JElXyTtFJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRnYKeZE+Sc0nmkxwecvw5ST6d5HKS28Y/piRplJFBTzIBHANuBXYBB5LsGlj2JeB24D3jHlCS1E2X7xTdDcxX1XmAJCeAfcCZBxZU1T29Y99dgRklSR10ueSyGbjQt73Q2ydJeghZ1TdFkxxMMpdkbnFxcTVfWpKa1yXoF4GtfdtbevuuWlUdr6qpqpqanJy8lqeQJF1Bl6DPAjuT7EiyEdgPTK/sWJKkqzUy6FV1GTgEnALOAier6nSSo0n2AiT5mSQLwK8Bb09yeiWHliQ9WJdPuVBVM8DMwL4jfY9nWboUI0laI94pKkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1IhOQU+yJ8m5JPNJDg85/kNJ3tc7/qkk28c+qSRpWSODnmQCOAbcCuwCDiTZNbDsJcB9VfUE4K3AG8c9qCRpeV3O0HcD81V1vqouASeAfQNr9gHv7D2+E3h+koxvTEnSKBs6rNkMXOjbXgCedaU1VXU5ydeBHwO+0r8oyUHgYG/zm0nOXcvQGmoTA/++H4ri/7utR/5ujtdNVzrQJehjU1XHgeOr+ZrrRZK5qppa6zmkQf5urp4ul1wuAlv7trf09g1dk2QD8Bjg3nEMKEnqpkvQZ4GdSXYk2QjsB6YH1kwDL+49vg34cFXV+MaUJI0y8pJL75r4IeAUMAHcUVWnkxwF5qpqGvgb4F1J5oGvshR9rS4vZemhyt/NVRJPpCWpDd4pKkmNMOiS1AiDLkmNWNXPoWs8kjyJpbtzN/d2XQSmq+rs2k0laa15hn6DSfIalv78QoB/7/0EeO+wP5wmPVQk+Z21nqF1fsrlBpPk88BTqur+gf0bgdNVtXNtJpOWl+RLVbVtredomZdcbjzfBX4C+OLA/sf3jklrJslnr3QIeNxqzrIeGfQbzx8AH0ryBb7/R9O2AU8ADq3VUFLP44BfAu4b2B/g31Z/nPXFoN9gquoDSZ7I0p817n9TdLaqvrN2k0kA/CPwqKq6e/BAko+u+jTrjNfQJakRfspFkhph0CWpEQZd60KSb444vj3Jf17lc74jyW3XN5k0PgZdkhph0LWuJHlUkg8l+XSSzyXp/8LzDUneneRskjuT/HDvn7k5yb8kuSvJqSSPX6PxpWUZdK033wJ+taqeCTwP+PMk6R37KeBtVfVk4BvA7yd5OPCXwG1VdTNwB/CnazC3NJKfQ9d6E+DPkjyHpTtrN/P9OxgvVNXHe4//FngF8AHgqcAHe92fAP53VSeWOjLoWm9+E5gEbq6q+5PcAzyid2zwpoxi6T8Ap6vq51ZvROnaeMlF681jgC/3Yv484Ka+Y9uSPBDu3wA+BpwDJh/Yn+ThSZ6yqhNLHRl0rTfvBqaSfA74beC/+o6dA16W5CzwWOCvquoScBvwxiSfAe4Gfn51R5a68dZ/SWqEZ+iS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmN+H+EBLWgkdNDHQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_data.groupby('label')['tfidf_pooing'].mean().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SIF Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> https://github.com/PrincetonML/SIF/blob/master/src/SIF_embedding.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TruncatedSVD(n_components=1, n_iter=7, random_state=0)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "svd = TruncatedSVD(n_components=1, n_iter=7, random_state=0)\n",
    "svd.fit(train_sentence1_tfidffeat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_siffeat = train_sentence1_tfidffeat - \\\n",
    "        train_sentence1_tfidffeat.dot(np.transpose(svd.components_)).dot(svd.components_)\n",
    "    \n",
    "train_sentence2_siffeat = train_sentence2_tfidffeat - \\\n",
    "        train_sentence2_tfidffeat.dot(np.transpose(svd.components_)).dot(svd.components_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_siffeat = normalize(train_sentence1_siffeat)\n",
    "train_sentence2_siffeat = normalize(train_sentence2_siffeat)\n",
    "\n",
    "sif_embedding_distance = [np.dot(x, y) for x, y in \n",
    "                        zip(train_sentence1_siffeat, train_sentence2_siffeat)]\n",
    "train_data['sif_embedding'] = sif_embedding_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='label'>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAONklEQVR4nO3dfYxdeV3H8feHKdXoEkLckWAfdhopYnmIsGPXhwRBIXazSathNV00sgZtjFTQNYZuNA2p0QBG+MNUQ6MbibKUdf8wo4w0ZAENKDizWMG2dpnUhU5jssOyQIiBbuHrH72Fy907c0/bOzPb37xfyST3nPPbe77ZNO89e+4901QVkqQb3zPWewBJ0ngYdElqhEGXpEYYdElqhEGXpEYYdElqxKb1OvHNN99cU1NT63V6SbohPfzww1+oqslhx9Yt6FNTU8zPz6/X6SXphpTkc8sd85aLJDXCoEtSIzoFPcmeJGeTLCQ5NOT4u5Kc7P08kuRLY59UkrSikffQk0wAR4HXAIvAXJKZqjp9ZU1V/U7f+t8CXrYKs0qSVtDlCn03sFBV56rqInAc2LfC+ruA941jOElSd12CvgU437e92Nv3FEluAXYAH77+0SRJV2PcH4ruBx6sqm8MO5jkQJL5JPNLS0tjPrUkbWxdgn4B2Na3vbW3b5j9rHC7paqOVdV0VU1PTg79Xrwk6Rp1ebBoDtiZZAeXQ74feN3goiQvBJ4D/NtYJ5Q01NShD6z3CE159G13rPcI123kFXpVXQIOAieAM8ADVXUqyZEke/uW7geOl38FkiSti06P/lfVLDA7sO/wwPZbxzeWJOlq+aSoJDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDWiU9CT7ElyNslCkkPLrPnFJKeTnEpy/3jHlCSNsmnUgiQTwFHgNcAiMJdkpqpO963ZCdwL/GRVPZHk+1drYEnScF2u0HcDC1V1rqouAseBfQNrfh04WlVPAFTVY+MdU5I0ysgrdGALcL5vexG4bWDNCwCSfByYAN5aVR8cy4TrbOrQB9Z7hKY8+rY71nsEqVldgt71fXYCrwS2Av+S5CVV9aX+RUkOAAcAtm/fPqZTS5Kg2y2XC8C2vu2tvX39FoGZqnqyqv4HeITLgf8OVXWsqqaranpycvJaZ5YkDdEl6HPAziQ7kmwG9gMzA2v+nstX5yS5mcu3YM6Nb0xJ0igjg15Vl4CDwAngDPBAVZ1KciTJ3t6yE8DjSU4DHwF+r6oeX62hJUlP1ekeelXNArMD+w73vS7gnt6PJGkd+KSoJDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDWiU9CT7ElyNslCkkNDjt+dZCnJyd7Pr41/VEnSSjaNWpBkAjgKvAZYBOaSzFTV6YGl76+qg6swoySpgy5X6LuBhao6V1UXgePAvtUdS5J0tboEfQtwvm97sbdv0GuTfDrJg0m2jWU6SVJn4/pQ9B+Aqap6KfAh4D3DFiU5kGQ+yfzS0tKYTi1Jgm5BvwD0X3Fv7e37lqp6vKq+3tv8S+DWYW9UVceqarqqpicnJ69lXknSMroEfQ7YmWRHks3AfmCmf0GS5/Vt7gXOjG9ESVIXI7/lUlWXkhwETgATwH1VdSrJEWC+qmaANyXZC1wCvgjcvYozS5KGGBl0gKqaBWYH9h3ue30vcO94R5MkXQ2fFJWkRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRnQKepI9Sc4mWUhyaIV1r01SSabHN6IkqYuRQU8yARwFbgd2AXcl2TVk3bOANwOfHPeQkqTRulyh7wYWqupcVV0EjgP7hqz7Q+DtwNfGOJ8kqaMuQd8CnO/bXuzt+5YkLwe2VdUHxjibJOkqXPeHokmeAbwT+N0Oaw8kmU8yv7S0dL2nliT16RL0C8C2vu2tvX1XPAt4MfDRJI8CPwbMDPtgtKqOVdV0VU1PTk5e+9SSpKfoEvQ5YGeSHUk2A/uBmSsHq+rLVXVzVU1V1RTwCWBvVc2vysSSpKFGBr2qLgEHgRPAGeCBqjqV5EiSvas9oCSpm01dFlXVLDA7sO/wMmtfef1jSZKulk+KSlIjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjOgU9yZ4kZ5MsJDk05PhvJPlMkpNJPpZk1/hHlSStZGTQk0wAR4HbgV3AXUOCfX9VvaSqfgR4B/DOcQ8qSVpZlyv03cBCVZ2rqovAcWBf/4Kq+krf5vcCNb4RJUldbOqwZgtwvm97EbhtcFGSNwL3AJuBnx7LdJKkzsb2oWhVHa2qHwTeAvzBsDVJDiSZTzK/tLQ0rlNLkugW9AvAtr7trb19yzkO/NywA1V1rKqmq2p6cnKy85CSpNG6BH0O2JlkR5LNwH5gpn9Bkp19m3cAnx3fiJKkLkbeQ6+qS0kOAieACeC+qjqV5AgwX1UzwMEkrwaeBJ4AXr+aQ0uSnqrLh6JU1SwwO7DvcN/rN495LknSVfJJUUlqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqRKegJ9mT5GyShSSHhhy/J8npJJ9O8lCSW8Y/qiRpJSODnmQCOArcDuwC7kqya2DZfwDTVfVS4EHgHeMeVJK0si5X6LuBhao6V1UXgePAvv4FVfWRqvq/3uYngK3jHVOSNEqXoG8BzvdtL/b2LecNwD9dz1CSpKu3aZxvluSXgWngp5Y5fgA4ALB9+/ZxnlqSNrwuV+gXgG1921t7+75DklcDvw/sraqvD3ujqjpWVdNVNT05OXkt80qSltEl6HPAziQ7kmwG9gMz/QuSvAx4N5dj/tj4x5QkjTIy6FV1CTgInADOAA9U1akkR5Ls7S37E+Am4O+SnEwys8zbSZJWSad76FU1C8wO7Dvc9/rVY55LknSVfFJUkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhrRKehJ9iQ5m2QhyaEhx1+R5FNJLiW5c/xjSpJGGRn0JBPAUeB2YBdwV5JdA8s+D9wN3D/uASVJ3WzqsGY3sFBV5wCSHAf2AaevLKiqR3vHvrkKM0qSOuhyy2ULcL5ve7G3T5L0NLKmH4omOZBkPsn80tLSWp5akprXJegXgG1921t7+65aVR2rqumqmp6cnLyWt5AkLaNL0OeAnUl2JNkM7AdmVncsSdLVGhn0qroEHAROAGeAB6rqVJIjSfYCJPnRJIvALwDvTnJqNYeWJD1Vl2+5UFWzwOzAvsN9r+e4fCtGkrROfFJUkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhph0CWpEQZdkhrRKehJ9iQ5m2QhyaEhx78ryft7xz+ZZGrsk0qSVjQy6EkmgKPA7cAu4K4kuwaWvQF4oqqeD7wLePu4B5UkrazLFfpuYKGqzlXVReA4sG9gzT7gPb3XDwI/kyTjG1OSNMqmDmu2AOf7theB25ZbU1WXknwZ+D7gC/2LkhwADvQ2v5rk7LUMraFuZuDf99NR/H+3jcg/m+N1y3IHugR9bKrqGHBsLc+5USSZr6rp9Z5DGuSfzbXT5ZbLBWBb3/bW3r6ha5JsAp4NPD6OASVJ3XQJ+hywM8mOJJuB/cDMwJoZ4PW913cCH66qGt+YkqRRRt5y6d0TPwicACaA+6rqVJIjwHxVzQB/BfxNkgXgi1yOvtaWt7L0dOWfzTUSL6QlqQ0+KSpJjTDoktQIgy5JjVjT76FrPJK8kMtP527p7boAzFTVmfWbStJ68wr9BpPkLVz+9QsB/r33E+B9w35xmvR0keRX13uG1vktlxtMkkeAF1XVkwP7NwOnqmrn+kwmrSzJ56tq+3rP0TJvudx4vgn8APC5gf3P6x2T1k2STy93CHjuWs6yERn0G89vAw8l+Szf/qVp24HnAwfXayip57nAzwJPDOwP8K9rP87GYtBvMFX1wSQv4PKvNe7/UHSuqr6xfpNJAPwjcFNVnRw8kOSjaz7NBuM9dElqhN9ykaRGGHRJaoRB14aQ5Ksjjk8l+a+rfM+/TnLn9U0mjY9Bl6RGGHRtKEluSvJQkk8l+UyS/r/wfFOS9yY5k+TBJN/T+2duTfLPSR5OciLJ89ZpfGlFBl0bzdeAn6+qlwOvAv40SXrHfgj486r6YeArwG8meSbwZ8CdVXUrcB/wR+swtzSS30PXRhPgj5O8gstP1m7h208wnq+qj/de/y3wJuCDwIuBD/W6PwH875pOLHVk0LXR/BIwCdxaVU8meRT47t6xwYcyisv/AThVVT++diNK18ZbLtpong081ov5q4Bb+o5tT3Il3K8DPgacBSav7E/yzCQvWtOJpY4Mujaa9wLTST4D/Arw333HzgJvTHIGeA7wF1V1EbgTeHuS/wROAj+xtiNL3fjovyQ1wit0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRvw/HilXUPdQCbUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_data.groupby('label')['sif_embedding'].mean().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean_pooing</th>\n",
       "      <th>max_pooing</th>\n",
       "      <th>tfidf_pooing</th>\n",
       "      <th>sif_embedding</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.818722</td>\n",
       "      <td>0.909529</td>\n",
       "      <td>0.718189</td>\n",
       "      <td>0.603005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.851903</td>\n",
       "      <td>0.919885</td>\n",
       "      <td>0.780175</td>\n",
       "      <td>0.692493</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      mean_pooing max_pooing tfidf_pooing sif_embedding\n",
       "             mean       mean         mean          mean\n",
       "label                                                  \n",
       "0        0.818722   0.909529     0.718189      0.603005\n",
       "1        0.851903   0.919885     0.780175      0.692493"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.groupby('label').agg({\n",
    "    'mean_pooing': ['mean'],\n",
    "    'max_pooing': ['mean'],\n",
    "    'tfidf_pooing': ['mean'],\n",
    "    'sif_embedding': ['mean']\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd = TruncatedSVD(n_components=3, n_iter=10, random_state=0)\n",
    "svd.fit(train_sentence1_tfidffeat)\n",
    "\n",
    "train_sentence1_usiffeat = train_sentence1_tfidffeat.copy()\n",
    "train_sentence2_usiffeat = train_sentence2_tfidffeat.copy()\n",
    "\n",
    "proj = lambda a, b: a.dot(b.transpose()) * b\n",
    "for i in range(3):\n",
    "    lambda_i = (svd.singular_values_[i] ** 2) / (svd.singular_values_ ** 2).sum()\n",
    "    pc = svd.components_[i]\n",
    "    train_sentence1_usiffeat = [ v_s - lambda_i * proj(v_s, pc) for v_s in train_sentence1_usiffeat]\n",
    "    train_sentence2_usiffeat = [ v_s - lambda_i * proj(v_s, pc) for v_s in train_sentence2_usiffeat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_usiffeat = np.vstack(train_sentence1_usiffeat)\n",
    "train_sentence2_usiffeat = np.vstack(train_sentence2_usiffeat)\n",
    "\n",
    "train_sentence1_tfidffeat = normalize(train_sentence1_usiffeat)\n",
    "train_sentence2_tfidffeat = normalize(train_sentence2_usiffeat)\n",
    "\n",
    "usif_embedding_distance = [np.dot(x, y) for x, y in \n",
    "                        zip(train_sentence1_usiffeat, train_sentence2_usiffeat)]\n",
    "train_data['usif_embedding'] = usif_embedding_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='label'>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAANV0lEQVR4nO3df4xdeVnH8feHlmoUQow7EuwPppGilh9Rdiz+SBAUYjebtBpW00UjGLQx0oDBGEo0G1OjAYwYY6qhwY1EwLLuH2aUSkNW0IBZ7CwuYLcWJnWhbUwYlhVCDHQrj3/0Llxmp72n7e3M9pn3K2lyzznf3vukmbx7eu49t6kqJEk3v6es9QCSpOkw6JLUhEGXpCYMuiQ1YdAlqQmDLklNbFyrF77llltqdnZ2rV5ekm5KDzzwwBeqamalY4OCnmQ38KfABuCdVfWWFdb8AvB7QAGfqKpXXek5Z2dnWVhYGPLykqSRJJ+93LGJQU+yATgMvAI4B5xIMl9VD42t2QG8GfiJqno0yfdc/9iSpKsx5Br6LmCxqs5U1QXgKLB32ZpfAw5X1aMAVfX56Y4pSZpkSNA3A2fHts+N9o17LvDcJB9Ncv/oEo0kaRVN603RjcAO4KXAFuBfkrygqv5nfFGS/cB+gG3btk3ppSVJMOwM/TywdWx7y2jfuHPAfFU9VlX/BXyaS4H/FlV1pKrmqmpuZmbFN2klSddoSNBPADuSbE+yCdgHzC9b83dcOjsnyS1cugRzZnpjSpImmRj0qroIHACOA6eAe6rqZJJDSfaMlh0HHknyEPAh4Ler6pEbNbQk6YmyVt+HPjc3V34OXZKuTpIHqmpupWNrdqeopOsze/D9az1CKw+/5fa1HuG6+V0uktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgYFPcnuJKeTLCY5uMLx1yRZSvLg6NevTn9USdKVbJy0IMkG4DDwCuAccCLJfFU9tGzp+6rqwA2YcU3NHnz/Wo/QysNvuX2tR5DaGnKGvgtYrKozVXUBOArsvbFjSZKu1pCgbwbOjm2fG+1b7pVJPpnk3iRbV3qiJPuTLCRZWFpauoZxJUmXM603Rf8emK2qFwIfBN610qKqOlJVc1U1NzMzM6WXliTBsKCfB8bPuLeM9n1DVT1SVV8bbb4TuHU640mShhoS9BPAjiTbk2wC9gHz4wuSPGtscw9wanojSpKGmPgpl6q6mOQAcBzYANxdVSeTHAIWqmoeeH2SPcBF4IvAa27gzJKkFUwMOkBVHQOOLdt319jjNwNvnu5okqSr4Z2iktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgYFPcnuJKeTLCY5eIV1r0xSSeamN6IkaYiJQU+yATgM3AbsBO5MsnOFdU8H3gB8bNpDSpImG3KGvgtYrKozVXUBOArsXWHd7wNvBb46xfkkSQMNCfpm4OzY9rnRvm9I8iJga1W9f4qzSZKuwnW/KZrkKcDbgd8asHZ/koUkC0tLS9f70pKkMUOCfh7YOra9ZbTvcU8Hng98OMnDwI8C8yu9MVpVR6pqrqrmZmZmrn1qSdITDAn6CWBHku1JNgH7gPnHD1bVl6rqlqqarapZ4H5gT1Ut3JCJJUkrmhj0qroIHACOA6eAe6rqZJJDSfbc6AElScNsHLKoqo4Bx5btu+sya196/WNJkq6Wd4pKUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaGBT0JLuTnE6ymOTgCsd/PcmnkjyY5CNJdk5/VEnSlUwMepINwGHgNmAncOcKwX5vVb2gqn4IeBvw9mkPKkm6siFn6LuAxao6U1UXgKPA3vEFVfXlsc3vBGp6I0qShtg4YM1m4OzY9jngxcsXJXkd8EZgE/BTU5lOkjTY1N4UrarDVfV9wJuA311pTZL9SRaSLCwtLU3rpSVJDAv6eWDr2PaW0b7LOQr87EoHqupIVc1V1dzMzMzgISVJkw0J+glgR5LtSTYB+4D58QVJdoxt3g58ZnojSpKGmHgNvaouJjkAHAc2AHdX1ckkh4CFqpoHDiR5OfAY8Cjw6hs5tCTpiYa8KUpVHQOOLdt319jjN0x5LknSVfJOUUlqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaGBT0JLuTnE6ymOTgCsffmOShJJ9Mcl+SZ09/VEnSlUwMepINwGHgNmAncGeSncuW/TswV1UvBO4F3jbtQSVJVzbkDH0XsFhVZ6rqAnAU2Du+oKo+VFX/O9q8H9gy3TElSZMMCfpm4OzY9rnRvst5LfCP1zOUJOnqbZzmkyX5JWAO+MnLHN8P7AfYtm3bNF9akta9IWfo54GtY9tbRvu+RZKXA78D7Kmqr630RFV1pKrmqmpuZmbmWuaVJF3GkKCfAHYk2Z5kE7APmB9fkOSHgXdwKeafn/6YkqRJJga9qi4CB4DjwCngnqo6meRQkj2jZX8EPA342yQPJpm/zNNJkm6QQdfQq+oYcGzZvrvGHr98ynNJkq6Sd4pKUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaGBT0JLuTnE6ymOTgCsdfkuTjSS4muWP6Y0qSJpkY9CQbgMPAbcBO4M4kO5ct+xzwGuC90x5QkjTMxgFrdgGLVXUGIMlRYC/w0OMLqurh0bGv34AZJUkDDLnkshk4O7Z9brRPkvQksqpviibZn2QhycLS0tJqvrQktTck6OeBrWPbW0b7rlpVHamquaqam5mZuZankCRdxpCgnwB2JNmeZBOwD5i/sWNJkq7WxKBX1UXgAHAcOAXcU1UnkxxKsgcgyY8kOQf8PPCOJCdv5NCSpCca8ikXquoYcGzZvrvGHp/g0qUYSdIa8U5RSWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpoYFPQku5OcTrKY5OAKx78tyftGxz+WZHbqk0qSrmhi0JNsAA4DtwE7gTuT7Fy27LXAo1X1HOBPgLdOe1BJ0pUNOUPfBSxW1ZmqugAcBfYuW7MXeNfo8b3ATyfJ9MaUJE2yccCazcDZse1zwIsvt6aqLib5EvDdwBfGFyXZD+wfbX4lyelrGVoruoVlf95PRvHfbuuRP5vT9ezLHRgS9KmpqiPAkdV8zfUiyUJVza31HNJy/myuniGXXM4DW8e2t4z2rbgmyUbgGcAj0xhQkjTMkKCfAHYk2Z5kE7APmF+2Zh549ejxHcA/VVVNb0xJ0iQTL7mMrokfAI4DG4C7q+pkkkPAQlXNA38J/HWSReCLXIq+VpeXsvRk5c/mKokn0pLUg3eKSlITBl2SmjDoktTEqn4OXdOR5Ae4dHfu5tGu88B8VZ1au6kkrTXP0G8ySd7Epa9fCPBvo18B/malL06TniyS/Mpaz9Cdn3K5yST5NPC8qnps2f5NwMmq2rE2k0lXluRzVbVtrefozEsuN5+vA98LfHbZ/meNjklrJsknL3cIeOZqzrIeGfSbz28C9yX5DN/80rRtwHOAA2s1lDTyTOBngEeX7Q/wr6s/zvpi0G8yVfWBJM/l0tcaj78peqKq/m/tJpMA+AfgaVX14PIDST686tOsM15Dl6Qm/JSLJDVh0CWpCYOudSHJVyYcn03yH1f5nH+V5I7rm0yaHoMuSU0YdK0rSZ6W5L4kH0/yqSTj/+H5xiTvSXIqyb1JvmP0e25N8s9JHkhyPMmz1mh86YoMutabrwI/V1UvAl4G/HGSjI59P/DnVfWDwJeB30jyVODPgDuq6lbgbuAP1mBuaSI/h671JsAfJnkJl+6s3cw372A8W1UfHT1+N/B64APA84EPjrq/AfjvVZ1YGsiga735RWAGuLWqHkvyMPDto2PLb8ooLv0FcLKqfmz1RpSujZdctN48A/j8KOYvA549dmxbksfD/SrgI8BpYObx/UmemuR5qzqxNJBB13rzHmAuyaeAXwb+c+zYaeB1SU4B3wX8RVVdAO4A3prkE8CDwI+v7sjSMN76L0lNeIYuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJamJ/wdSzwJaclK9xwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_data.groupby('label')['usif_embedding'].mean().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean_pooing</th>\n",
       "      <th>max_pooing</th>\n",
       "      <th>tfidf_pooing</th>\n",
       "      <th>sif_embedding</th>\n",
       "      <th>usif_embedding</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.818722</td>\n",
       "      <td>0.909529</td>\n",
       "      <td>0.718189</td>\n",
       "      <td>0.603005</td>\n",
       "      <td>0.507293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.851903</td>\n",
       "      <td>0.919885</td>\n",
       "      <td>0.780175</td>\n",
       "      <td>0.692493</td>\n",
       "      <td>0.584793</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      mean_pooing max_pooing tfidf_pooing sif_embedding usif_embedding\n",
       "             mean       mean         mean          mean           mean\n",
       "label                                                                 \n",
       "0        0.818722   0.909529     0.718189      0.603005       0.507293\n",
       "1        0.851903   0.919885     0.780175      0.692493       0.584793"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.groupby('label').agg({\n",
    "    'mean_pooing': ['mean'],\n",
    "    'max_pooing': ['mean'],\n",
    "    'tfidf_pooing': ['mean'],\n",
    "    'sif_embedding': ['mean'],\n",
    "    'usif_embedding': ['mean']\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11, 100)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sentence1_feat[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_mean(vals, p):\n",
    "    p = float(p)\n",
    "    return np.power(\n",
    "        np.mean(\n",
    "            np.power(\n",
    "                np.array(vals, dtype=complex),\n",
    "                p),\n",
    "            axis=0),\n",
    "        1 / p\n",
    "    )\n",
    "\n",
    "def get_sentence_embedding(word_embeddings):\n",
    "    concat_embs = []\n",
    "    if len(word_embeddings) == 0:\n",
    "        return np.zeros(500)\n",
    "    concat_embs += [np.mean(word_embeddings, axis=0)]\n",
    "    concat_embs += [np.max(word_embeddings, axis=0)]\n",
    "    concat_embs += [np.min(word_embeddings, axis=0)]\n",
    "    concat_embs += [gen_mean(word_embeddings, p=2.0).real]\n",
    "    concat_embs += [gen_mean(word_embeddings, p=3.0).real]\n",
    "    sentence_embedding = np.concatenate(\n",
    "                concat_embs,\n",
    "                axis=0\n",
    "            )\n",
    "    return sentence_embedding\n",
    "\n",
    "train_sentence1_pmeansfeat = [get_sentence_embedding(feat) for feat in train_sentence1_feat]\n",
    "train_sentence2_pmeansfeat = [get_sentence_embedding(feat) for feat in train_sentence2_feat]\n",
    "\n",
    "train_sentence1_pmeansfeat = np.vstack(train_sentence1_pmeansfeat)\n",
    "train_sentence2_pmeansfeat = np.vstack(train_sentence2_pmeansfeat)\n",
    "\n",
    "train_sentence1_pmeansfeat = normalize(train_sentence1_pmeansfeat)\n",
    "train_sentence2_pmeansfeat = normalize(train_sentence2_pmeansfeat)\n",
    "\n",
    "pmeans_embedding_distance = [np.dot(x, y) for x, y in \n",
    "                        zip(train_sentence1_pmeansfeat, train_sentence2_pmeansfeat)]\n",
    "train_data['pmeans_embedding'] = pmeans_embedding_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean_pooing</th>\n",
       "      <th>max_pooing</th>\n",
       "      <th>tfidf_pooing</th>\n",
       "      <th>sif_embedding</th>\n",
       "      <th>usif_embedding</th>\n",
       "      <th>pmeans_embedding</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.818722</td>\n",
       "      <td>0.909529</td>\n",
       "      <td>0.718189</td>\n",
       "      <td>0.603005</td>\n",
       "      <td>0.507293</td>\n",
       "      <td>0.910251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.851903</td>\n",
       "      <td>0.919885</td>\n",
       "      <td>0.780175</td>\n",
       "      <td>0.692493</td>\n",
       "      <td>0.584793</td>\n",
       "      <td>0.921331</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      mean_pooing max_pooing tfidf_pooing sif_embedding usif_embedding  \\\n",
       "             mean       mean         mean          mean           mean   \n",
       "label                                                                    \n",
       "0        0.818722   0.909529     0.718189      0.603005       0.507293   \n",
       "1        0.851903   0.919885     0.780175      0.692493       0.584793   \n",
       "\n",
       "      pmeans_embedding  \n",
       "                  mean  \n",
       "label                   \n",
       "0             0.910251  \n",
       "1             0.921331  "
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.groupby('label').agg({\n",
    "    'mean_pooing': ['mean'],\n",
    "    'max_pooing': ['mean'],\n",
    "    'tfidf_pooing': ['mean'],\n",
    "    'sif_embedding': ['mean'],\n",
    "    'usif_embedding': ['mean'],\n",
    "    'pmeans_embedding': ['mean']\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## doc2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(train_sentence1+train_sentence2)]\n",
    "model = Doc2Vec(documents, vector_size=100, window=5, min_count=1, workers=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_doc2feat = [model.infer_vector(x) for x in train_sentence1]\n",
    "train_sentence2_doc2feat = [model.infer_vector(x) for x in train_sentence2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentence1_doc2feat = np.vstack(train_sentence1_doc2feat)\n",
    "train_sentence2_doc2feat = np.vstack(train_sentence2_doc2feat)\n",
    "\n",
    "train_sentence1_doc2feat = normalize(train_sentence1_doc2feat)\n",
    "train_sentence2_doc2feat = normalize(train_sentence2_doc2feat)\n",
    "\n",
    "doc2vec_distance = [np.dot(x, y) for x, y in \n",
    "                        zip(train_sentence1_doc2feat, train_sentence2_doc2feat)]\n",
    "train_data['doc2vec'] = doc2vec_distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean_pooing</th>\n",
       "      <th>max_pooing</th>\n",
       "      <th>tfidf_pooing</th>\n",
       "      <th>sif_embedding</th>\n",
       "      <th>doc2vec</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.824265</td>\n",
       "      <td>0.910736</td>\n",
       "      <td>0.720056</td>\n",
       "      <td>0.604179</td>\n",
       "      <td>1.087763e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.856910</td>\n",
       "      <td>0.921142</td>\n",
       "      <td>0.781910</td>\n",
       "      <td>0.693707</td>\n",
       "      <td>-1.096150e-04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      mean_pooing max_pooing tfidf_pooing sif_embedding       doc2vec\n",
       "             mean       mean         mean          mean          mean\n",
       "label                                                                \n",
       "0        0.824265   0.910736     0.720056      0.604179  1.087763e-07\n",
       "1        0.856910   0.921142     0.781910      0.693707 -1.096150e-04"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.groupby('label').agg({\n",
    "    'mean_pooing': ['mean'],\n",
    "    'max_pooing': ['mean'],\n",
    "    'tfidf_pooing': ['mean'],\n",
    "    'sif_embedding': ['mean'],\n",
    "    'doc2vec': ['mean']\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 序列模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN\n",
    "\n",
    "https://pytorch.org/docs/stable/generated/torch.nn.RNN.html#torch.nn.RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input_size, hidden_size, num_layers\n",
    "model = nn.RNN(10, 20, 2)\n",
    "\n",
    "# sequence length, batch size, input_size\n",
    "input = torch.randn(10, 1, 10)\n",
    "\n",
    "output, hn = model(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([10, 1, 20]), torch.Size([2, 1, 20]))"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.shape, hn.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM\n",
    "\n",
    "https://pytorch.org/docs/stable/generated/torch.nn.LSTM.html#torch.nn.LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input_size, hidden_size, num_layers\n",
    "model = nn.LSTM(10, 20, 2)\n",
    "input = torch.randn(5, 3, 10)\n",
    "\n",
    "output, (hn, cn) = model(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([5, 3, 20]), torch.Size([2, 3, 20]), torch.Size([2, 3, 20]))"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.shape, hn.shape, cn.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc-autonumbering": false,
  "toc-showmarkdowntxt": false,
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
